
#
# Loading Necessary Libraries
#

# To help with reading and manipulating data
import pandas as pd
import numpy as np

# To help with data visualization
%matplotlib inline
import matplotlib.pyplot as plt
import seaborn as sns
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import animation

sns.set(
    color_codes=True
)  # -----This adds a background color to all the plots created using seaborn

# Allow the use of Display via interactive Python
from IPython.display import display

# Import tabulate. library used for creating tables in a visually appealing format.
from tabulate import tabulate

# to scale the data using z-score
from sklearn.preprocessing import StandardScaler

# to compute distances
from scipy.spatial.distance import cdist, pdist

# to perform k-means clustering and compute silhouette scores
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score

# to visualize the elbow curve and silhouette scores
from yellowbrick.cluster import (
    KElbowVisualizer,
    SilhouetteVisualizer,
    InterclusterDistance,
)


# to perform hierarchical clustering, compute cophenetic correlation, and create dendrograms
from sklearn.cluster import AgglomerativeClustering
from scipy.cluster.hierarchy import dendrogram, linkage, cophenet

# For Dimensionality Reduction
from sklearn.decomposition import PCA

# To define maximum number of columns to be displayed in a dataframe
pd.set_option("display.max_columns", None)

# To supress scientific notations for a dataframe
pd.set_option("display.float_format", lambda x: "%.3f" % x)

# To supress warnings
import warnings

warnings.filterwarnings("ignore")

# Making the Python code more structured automatically
%load_ext nb_black

print("Loading Libraries... Done.")

Loading Libraries... Done.


# Loading Dataset
data_path = "Credit+Card+Customer+Data.xlsx"
data = pd.read_excel(data_path)

# Making a copy of the data to avoid any changes to original data
df = data.copy()

print("Loading Dataset... Done.")

Loading Dataset... Done.


# Checking the top 5 and bottom 5

display(df.head())  # -----looking at head (top 5 observations)
display(df.tail())  # -----looking at tail (bottom 5 observations)


# -----Print the dimension of the data
print(f"There are {df.shape[0]} rows and {df.shape[1]} columns")

There are 660 rows and 7 columns


# -----Displaying information about features of the Dataset
df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 660 entries, 0 to 659
Data columns (total 7 columns):
 #   Column               Non-Null Count  Dtype
---  ------               --------------  -----
 0   Sl_No                660 non-null    int64
 1   Customer Key         660 non-null    int64
 2   Avg_Credit_Limit     660 non-null    int64
 3   Total_Credit_Cards   660 non-null    int64
 4   Total_visits_bank    660 non-null    int64
 5   Total_visits_online  660 non-null    int64
 6   Total_calls_made     660 non-null    int64
dtypes: int64(7)
memory usage: 36.2 KB


# -----Displaying Statistical Summary of Numerical Data
df.describe().T


# Checking missing values across each columns

c_missing = pd.Series(df.isnull().sum(), name="Missing Count")  # -----Count Missing

p_missing = pd.Series(
    round(df.isnull().sum() / df.shape[0] * 100, 2), name="% Missing"
)  # -----Percentage Missing


# Combine into 1 Dataframe
missing_df = pd.concat([c_missing, p_missing], axis=1)

# # Display missing info
# display(missing_df)

missing_df.sort_values(by="% Missing", ascending=False).style.background_gradient(
    cmap="YlOrRd"
)


# Checking for duplicate records

df.duplicated().sum()

0


# checking the number of unique values in each column
df.nunique()

Sl_No                  660
Customer Key           655
Avg_Credit_Limit       110
Total_Credit_Cards      10
Total_visits_bank        6
Total_visits_online     16
Total_calls_made        11
dtype: int64


# getting the count for each unique value in Customer Key
filtered = df.groupby("Customer Key").count()

for i in filtered.loc[filtered.Sl_No >= 2].index:
    display(df.loc[df["Customer Key"] == i])


# -----
# User defined function to plot labeled_barplot
# -----


def labeled_barplot(data, feature, perc=False, v_ticks=True, n=None):
    """
    Barplot with percentage at the top

    data: dataframe
    feature: dataframe column
    perc: whether to display percentages instead of count (default is False)
    n: displays the top n category levels (default is None, i.e., display all levels)
    """

    total = len(data[feature])  # length of the column
    count = data[feature].nunique()
    if n is None:
        plt.figure(figsize=(count + 1, 5))
    else:
        plt.figure(figsize=(n + 1, 5))

    if v_ticks is True:
        plt.xticks(rotation=90)

    ax = sns.countplot(
        data=data,
        x=feature,
        palette="Paired",
        order=data[feature].value_counts().index[:n].sort_values(),
    )

    for p in ax.patches:
        if perc == True:
            label = "{:.1f}%".format(
                100 * p.get_height() / total
            )  # percentage of each class of the category
        else:
            label = p.get_height()  # count of each level of the category

        x = p.get_x() + p.get_width() / 2  # width of the plot
        y = p.get_height()  # height of the plot

        ax.annotate(
            label,
            (x, y),
            ha="center",
            va="center",
            size=12,
            xytext=(0, 5),
            textcoords="offset points",
        )  # annotate the percentage
    plt.show()  # show the plot


# -----
# User defined function to prints the 5 point summary and histogram, box plot,
#   and cumulative density distribution plots
# -----


def summary(data, x):
    """
    The function prints the 5 point summary and histogram, box plot,
    and cumulative density distribution plots for each
    feature name passed as the argument.

    Parameters:
    ----------

    x: str, feature name

    Usage:
    ------------

    summary('age')
    """

    x_min = data[x].min()
    x_max = data[x].max()
    Q1 = data[x].quantile(0.25)
    Q2 = data[x].quantile(0.50)
    Q3 = data[x].quantile(0.75)

    dict = {"Min": x_min, "Q1": Q1, "Q2": Q2, "Q3": Q3, "Max": x_max}
    ldf = pd.DataFrame(data=dict, index=["Value"])
    print(f"5 Point Summary of {x.capitalize()} Attribute:\n")
    print(tabulate(ldf, headers="keys", tablefmt="psql"))

    fig, axs = plt.subplots(nrows=3, ncols=1, figsize=(16, 22))
    sns.set_palette("Pastel1")

    # Histogram
    ax1 = sns.distplot(data[x], color="purple", ax=axs[0])
    ax1.axvline(np.mean(data[x]), color="purple", linestyle="--")
    ax1.axvline(np.median(data[x]), color="black", linestyle="-")
    ax1.set_title(f"{x.capitalize()} Density Distribution")

    # Boxplot
    ax2 = sns.boxplot(
        x=data[x], palette="cool", width=0.7, linewidth=0.6, showmeans=True, ax=axs[1]
    )
    ax2.set_title(f"{x.capitalize()} Boxplot")

    # Cummulative plot
    ax3 = sns.kdeplot(data[x], cumulative=True, linewidth=1.5, ax=axs[2])
    ax3.set_title(f"{x.capitalize()} Cumulative Density Distribution")

    plt.subplots_adjust(hspace=0.4)
    plt.show()


# Loop through all the variables of interest and perform a univariate statistical summary
for col in df.columns[2:]:
    print("=" * 115, "\n\n")
    summary(df, col)

=================================================================================================================== 


5 Point Summary of Avg_credit_limit Attribute:

+-------+-------+-------+-------+-------+--------+
|       |   Min |    Q1 |    Q2 |    Q3 |    Max |
|-------+-------+-------+-------+-------+--------|
| Value |  3000 | 10000 | 18000 | 48000 | 200000 |
+-------+-------+-------+-------+-------+--------+

=================================================================================================================== 


5 Point Summary of Total_credit_cards Attribute:

+-------+-------+------+------+------+-------+
|       |   Min |   Q1 |   Q2 |   Q3 |   Max |
|-------+-------+------+------+------+-------|
| Value |     1 |    3 |    5 |    6 |    10 |
+-------+-------+------+------+------+-------+

=================================================================================================================== 


5 Point Summary of Total_visits_bank Attribute:

+-------+-------+------+------+------+-------+
|       |   Min |   Q1 |   Q2 |   Q3 |   Max |
|-------+-------+------+------+------+-------|
| Value |     0 |    1 |    2 |    4 |     5 |
+-------+-------+------+------+------+-------+

=================================================================================================================== 


5 Point Summary of Total_visits_online Attribute:

+-------+-------+------+------+------+-------+
|       |   Min |   Q1 |   Q2 |   Q3 |   Max |
|-------+-------+------+------+------+-------|
| Value |     0 |    1 |    2 |    4 |    15 |
+-------+-------+------+------+------+-------+

=================================================================================================================== 


5 Point Summary of Total_calls_made Attribute:

+-------+-------+------+------+------+-------+
|       |   Min |   Q1 |   Q2 |   Q3 |   Max |
|-------+-------+------+------+------+-------|
| Value |     0 |    1 |    3 |    5 |    10 |
+-------+-------+------+------+------+-------+


for col in df.columns[3:]:
    labeled_barplot(df, col, perc=True)


# Define the Correlation Matrix
cm = df.iloc[:, 2:]

# Display the matrix
plt.figure(figsize=(15, 7))
sns.heatmap(cm.corr(), annot=True, vmin=-1, vmax=1, fmt=".2f", cmap="Spectral")
plt.show()


# Create a pairplot

sns.pairplot(data=cm, diag_kind="kde", plot_kws={"color": "purple"})

plt.show()


sns.pairplot(
    data=df[
        [
            "Total_visits_bank",
            "Total_visits_online",
            "Total_calls_made",
            "Total_Credit_Cards",
        ]
    ],
    hue="Total_Credit_Cards",
)

plt.show()


# Create a 3D Plot that shows the relationship of how the customers connect with the bank

# Create figure and axis
fig = plt.figure(figsize=(10, 10))
ax = plt.axes(projection="3d")

# Initialize data
x = df["Total_visits_bank"]
y = df["Total_visits_online"]
z = df["Total_calls_made"]

# Create a 3D scatter plot
ax.scatter(x, y, z, c=z, cmap="viridis")

# Set labels and title
ax.set_xlabel("Total_visits_bank")
ax.set_ylabel("Total_visits_online")
ax.set_zlabel("Total_calls_made")
ax.set_title("3D Scatter Plot")

# Set the azimuthal angle
ax.view_init(azim=60)

# Display the plot
plt.show()


########################################################################
# The code commented out below was used to generate the gif of the 3D plot.
# The gif was embadded as a markdown independently.
########################################################################


########################################################################
# fig = plt.figure(figsize=(10, 10))

# ax = Axes3D(fig)


# # Initialize data
# x = df["Total_visits_bank"]
# y = df["Total_visits_online"]
# z = df["Total_calls_made"]


# g = ax.scatter(x, y, z, c=x, marker="o", depthshade=False, cmap="Paired")
# ax.set_xlabel("Total Bank Visits")
# ax.set_ylabel("Total Visits Online")
# ax.set_zlabel("Total Calls Made")

# # produce a legend with the unique colors from the scatter
# legend = ax.legend(
#     *g.legend_elements(),
#     loc="lower center",
#     title="Total bank visits",
#     borderaxespad=-10,
#     ncol=4
# )
# ax.add_artist(legend)

# def rotate(angle):
#     ax.view_init(azim=angle)


# angle = 1
# ani = animation.FuncAnimation(fig, rotate, frames=np.arange(0, 360, angle), interval=1)
# ani.save("Cluster_plot.gif", writer=animation.PillowWriter(fps=25))
########################################################################


# Make a copy of the dataframe and drop Sl_No & Customer Key
df2 = df.copy()
df.drop(["Sl_No", "Customer Key"], axis=1, inplace=True)


# Create an instance of StandardScaler
scaler = StandardScaler()

# Fit and transform the DataFrame
scaled_df = pd.DataFrame(scaler.fit_transform(df), columns=df.columns)


display(scaled_df.head())


# Make copies for K-Means and Hierarchical Clustering.
km_df = scaled_df.copy()
hc_df = scaled_df.copy()


######
# Code to generate the Inertia Values, and create the Elbow Plot with those values.
######

# List to store cluster and intra cluster distance
clusters = []
# Variance - Intra cluster sum of distance ; within cluster sum of distance
inertia_vals = [] 

# Multiple values of K are utilized to come up with the optimum cluster value
# Cluster number and intra cluster distance is appended for plotting the elbow curve
for k in range(1, 11, 1):

    # train clustering with the specified K
    model = KMeans(n_clusters=k, random_state=1)
    model.fit(km_df)

    # append model to cluster list
    clusters.append(model)
    # append inertia to list
    inertia_vals.append(model.inertia_)
 
print("Inertia Values:")
display(inertia_vals)   # ---Intra cluster distance


# Plot the inertia vs K values
plt.figure(figsize=(10, 6))
plt.plot(range(1, 11, 1), inertia_vals, marker="*")
plt.title("Sum of Intra Cluster Distance across different values of K")

plt.axvline(3, linestyle="--")

plt.xlabel("K - clusters")
plt.ylabel("Inertia (Intra Cluster Distance)")

plt.xticks(range(1, 11, 1))

plt.show()

Inertia Values:

[3300.0,
 2040.989816478495,
 933.0437490000536,
 780.7736895551767,
 704.4759188657506,
 642.4285451423209,
 600.2238778375961,
 574.4418958177624,
 538.8269188945017,
 509.1631378891258]


# Using KElbowVisualizer fromYellowBrick
model = KMeans(random_state=1)
visualizer = KElbowVisualizer(model, k=(2, 10), timings=True)
visualizer.fit(km_df)  # fit the data to the visualizer
visualizer.show()  # finalize and render figure

<AxesSubplot:title={'center':'Distortion Score Elbow for KMeans Clustering'}, xlabel='k', ylabel='distortion score'>


#####
# Code to calculate the Silhouette Scores
#####
silhouette = []

for i in range(1, 10, 1):

    print("~" * 40)
    print(clusters[i])
    print("Silhouette score:", silhouette_score(km_df, clusters[i].predict(km_df)))
    silhouette.append(round(silhouette_score(km_df, clusters[i].predict(km_df)), 3))

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
KMeans(n_clusters=2, random_state=1)
Silhouette score: 0.41842496663230405
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
KMeans(n_clusters=3, random_state=1)
Silhouette score: 0.5157182558882754
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
KMeans(n_clusters=4, random_state=1)
Silhouette score: 0.35566706193773706
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
KMeans(n_clusters=5, random_state=1)
Silhouette score: 0.2717470361094591
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
KMeans(n_clusters=6, random_state=1)
Silhouette score: 0.25590676529850875
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
KMeans(n_clusters=7, random_state=1)
Silhouette score: 0.2479864465613871
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
KMeans(random_state=1)
Silhouette score: 0.2414240144772954
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
KMeans(n_clusters=9, random_state=1)
Silhouette score: 0.2184645050766369
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
KMeans(n_clusters=10, random_state=1)
Silhouette score: 0.2140658673097117


#####
# Code to plot the Silhouette Scores
#####
plt.figure(figsize=(10, 5))
plt.plot(range(2, 11, 1), silhouette, marker="*")
plt.title("Sum of Intra Cluster Distance across different values of K")
plt.axvline(3, linestyle="--")

plt.xlabel("K - clusters")
plt.ylabel("Silhouette Score")
plt.xticks(range(2, 11, 1))
plt.show()


# Silhouette Visualizer for K=3

plt.figure(figsize=(15, 8))
visualizer = SilhouetteVisualizer(KMeans(3, random_state=1))
visualizer.fit(km_df)
visualizer.show()

<AxesSubplot:title={'center':'Silhouette Plot of KMeans Clustering for 660 Samples in 3 Centers'}, xlabel='silhouette coefficient values', ylabel='cluster label'>


# Silhouette Visualizer for K=4

plt.figure(figsize=(15, 8))
visualizer = SilhouetteVisualizer(KMeans(4, random_state=1))
visualizer.fit(km_df)
visualizer.show()

<AxesSubplot:title={'center':'Silhouette Plot of KMeans Clustering for 660 Samples in 4 Centers'}, xlabel='silhouette coefficient values', ylabel='cluster label'>


# Silhouette Visualizer for K=5

plt.figure(figsize=(15, 8))
visualizer = SilhouetteVisualizer(KMeans(5, random_state=1))
visualizer.fit(km_df)
visualizer.show()

<AxesSubplot:title={'center':'Silhouette Plot of KMeans Clustering for 660 Samples in 5 Centers'}, xlabel='silhouette coefficient values', ylabel='cluster label'>


#####
# Code to visualise the Intercluster Distance
#####

plt.figure(figsize=(15, 8))

# Instantiate the clustering model and visualizer
model = KMeans(3, random_state=1)

visualizer = InterclusterDistance(model, random_state=1)

# Fit the data to the visualizer
visualizer.fit(km_df)

# Finalize and render the figure
visualizer.show()

<AxesSubplot:title={'center':'KMeans Intercluster Distance Map (via MDS)'}, xlabel='PC2', ylabel='PC1'>


%%time
kmeans = KMeans(n_clusters=3, random_state=1)
kmeans.fit(km_df)

CPU times: user 122 ms, sys: 39.9 ms, total: 162 ms
Wall time: 37.4 ms

KMeans(n_clusters=3, random_state=1)


# adding kmeans cluster labels
df["K_means_segments"] = kmeans.labels_
df2["K_means_segments"] = kmeans.labels_
km_df["K_means_segments"] = kmeans.labels_

km_df["K_means_segments"].value_counts()

1    386
0    224
2     50
Name: K_means_segments, dtype: int64


#####
# Dimensionality reduction whereby we convert the dataset from 5 columns to 2 columns
# where each of these columns are combined to created pc1, pc2.
#####

pca = PCA(n_components=2, random_state=1)

principalComponents = pca.fit_transform(km_df)

X_std = pd.DataFrame(data=principalComponents, columns=["pc1", "pc2"])

X_std.head()


#####
# Code to check the amount of variance explained by the principal components
# (the information captured by the PCs)
#####

variance = pca.explained_variance_ratio_

print("Variance explained by each Principal Component:")

total = 0
j = 1
for i in variance:
    print("PC", j, ":", round(i * 100, 2))
    j += 1
    total += i

# Format the statement with the value of 'total'
statement = (
    "The 2 Principal Components explain {}% of the information in the data".format(
        round(total * 100, 2)
    )
)
# Print the statement
print("\n")
print(statement)

Variance explained by each Principal Component:
PC 1 : 48.54
PC 2 : 35.01


The 2 Principal Components explain 83.55% of the information in the data


# Plot in 2D

X_std["labels"] = kmeans.labels_

plt.figure(figsize=(10, 8))

sns.scatterplot(x=X_std.pc1, y=X_std.pc2, hue=X_std.labels)

<AxesSubplot:xlabel='pc1', ylabel='pc2'>


#####
# Code to visualise the Boxplots of numerical variables for each cluster obtained using K-means Clustering
#####

fig, axes = plt.subplots(1, 5, figsize=(16, 6))
fig.suptitle(
    "Boxplot of numerical variables for each cluster obtained using K-means Clustering",
    fontsize=20,
)

for ii in range(5):
    sns.boxplot(ax=axes[ii], y=df[df.columns[ii]], x=df["K_means_segments"])

fig.tight_layout(pad=2.0)


#####
# Code to map each Cluster to a Profile
# C1=Cluster 1, C2=Cluster 2, C3=Cluster 3
#####


def cluster_map(val):

    if val == 2:

        return "C1-OnlineBankers - Customers with: High Credit Limit & High Total Credit Cards"

    elif val == 1:

        return "C2-TelephoneBankers - Customers with: Moderate Credit Limit & Moderate Total Credit Cards"

    else:

        return "C3-PhysicalBankers - Customers with: Low Credit Limit & Low Total Credit Cards"


#Apply the Profiles to the Cluster Segments

df["K_means_Group"] = df["K_means_segments"].apply(cluster_map)

pd.set_option('display.max_colwidth', None)

df.groupby("K_means_Group").mean().round(2).reset_index().sort_values(by=["Total_Credit_Cards"], ascending=False)


# Calculate the Cophenetic Coefficient for various pairs of distance metric and linkage methods.

# list of distance metrics
distance_metrics = ["euclidean", "mahalanobis", "cityblock", "cosine", "correlation"]

# list of linkage methods
linkage_methods = ["single", "complete", "average", "weighted"]

high_cophenet_corr = 0  # ---Stores the highest cophenetic coefficient

high_dm_lm = [0, 0]

for dm in distance_metrics:

    print("-" * 50)

    for lm in linkage_methods:

        # Calulcate the dendrogram using the linkage
        Z = linkage(hc_df, metric=dm, method=lm)

        c, coph_dists = cophenet(Z, pdist(hc_df))

        print(
            "Cophenetic correlation for {} distance and {} linkage is {}.".format(
                dm.capitalize(), lm.capitalize(), round(c, 3)
            )
        )

        if high_cophenet_corr < c:
            high_cophenet_corr = c
            high_dm_lm[0] = dm
            high_dm_lm[1] = lm

--------------------------------------------------
Cophenetic correlation for Euclidean distance and Single linkage is 0.739.
Cophenetic correlation for Euclidean distance and Complete linkage is 0.86.
Cophenetic correlation for Euclidean distance and Average linkage is 0.898.
Cophenetic correlation for Euclidean distance and Weighted linkage is 0.886.
--------------------------------------------------
Cophenetic correlation for Mahalanobis distance and Single linkage is 0.706.
Cophenetic correlation for Mahalanobis distance and Complete linkage is 0.666.
Cophenetic correlation for Mahalanobis distance and Average linkage is 0.833.
Cophenetic correlation for Mahalanobis distance and Weighted linkage is 0.781.
--------------------------------------------------
Cophenetic correlation for Cityblock distance and Single linkage is 0.725.
Cophenetic correlation for Cityblock distance and Complete linkage is 0.873.
Cophenetic correlation for Cityblock distance and Average linkage is 0.896.
Cophenetic correlation for Cityblock distance and Weighted linkage is 0.883.
--------------------------------------------------
Cophenetic correlation for Cosine distance and Single linkage is 0.747.
Cophenetic correlation for Cosine distance and Complete linkage is 0.532.
Cophenetic correlation for Cosine distance and Average linkage is 0.672.
Cophenetic correlation for Cosine distance and Weighted linkage is 0.566.
--------------------------------------------------
Cophenetic correlation for Correlation distance and Single linkage is 0.61.
Cophenetic correlation for Correlation distance and Complete linkage is 0.489.
Cophenetic correlation for Correlation distance and Average linkage is 0.66.
Cophenetic correlation for Correlation distance and Weighted linkage is 0.532.


# printing the combination of distance metric and linkage method with the highest cophenetic correlation
print(
    "Highest cophenetic correlation is {}, which is obtained with {} distance and {} linkage.".format(
        high_cophenet_corr, high_dm_lm[0].capitalize(), high_dm_lm[1].capitalize()
    )
)

Highest cophenetic correlation is 0.8977080867389372, which is obtained with Euclidean distance and Average linkage.


# Calculate the Cophenetic Coefficient for Euclidean distance and various linkage methods.

# list of linkage methods
linkage_methods = [
    "single",
    "complete",
    "average",
    "weighted",
    "ward",  # Ward only works with Euclidean distance
]

high_cophenet_corr = 0
high_dm_lm = [0, 0]

for lm in linkage_methods:
    Z = linkage(hc_df, metric="euclidean", method=lm)
    c, coph_dists = cophenet(Z, pdist(hc_df))
    print(
        "Cophenetic correlation for Euclidean distance & {} linkage is {}.".format(
            lm.capitalize(), c
        )
    )
    if high_cophenet_corr < c:
        high_cophenet_corr = c
        high_dm_lm[0] = "euclidean"
        high_dm_lm[1] = lm

Cophenetic correlation for Euclidean distance & Single linkage is 0.7391220243806552.
Cophenetic correlation for Euclidean distance & Complete linkage is 0.8599730607972423.
Cophenetic correlation for Euclidean distance & Average linkage is 0.8977080867389372.
Cophenetic correlation for Euclidean distance & Weighted linkage is 0.8861746814895477.
Cophenetic correlation for Euclidean distance & Ward linkage is 0.7415156284827493.


# Printing the linkage method with the highest cophenetic correlation
print()
print(
    "Highest cophenetic correlation for Euclidean distance is {}, which is obtained with {} linkage.".format(
        high_cophenet_corr, high_dm_lm[1].capitalize()
    )
)

Highest cophenetic correlation for Euclidean distance is 0.8977080867389372, which is obtained with Average linkage.


# list of linkage methods
linkage_methods = ["single", "complete", "average", "weighted", "ward"]

results_cophenetic_coef = []

for i in linkage_methods:

    plt.figure(figsize=(20, 8))

    plt.ylabel("Dendrogram Distance")

    Z = linkage(hc_df, i, metric="euclidean")

    cc, cophn_dist = cophenet(Z, pdist(hc_df))

    dendrogram(
        Z,
        leaf_rotation=90.0,
        p=7,
        leaf_font_size=10,
        truncate_mode="level",
        color_threshold=60,
        link_color_func=lambda k: "purple",
    )

    plt.tight_layout()

    plt.title(
        "Linkage Type: " + i + " having cophenetic coefficient : " + str(round(cc, 3)),
        fontsize=20,
    )

    plt.show()
    results_cophenetic_coef.append((i, cc))
    print(i, cc)

single 0.7391220243806552

complete 0.8599730607972423

average 0.8977080867389372

weighted 0.8861746814895477

ward 0.7415156284827493


# Create and display a dataframe to compare cophenetic correlations for different linkage methods for euclidean distance
results_cophenetic_coef_df = pd.DataFrame(
    results_cophenetic_coef, columns=["LinkageMethod", "CopheneticCoefficient"]
)
results_cophenetic_coef_df


#####
# Code to calculate the Silhouette Scores
#####

sil_score = []

for i in range(2, 11):

    model = AgglomerativeClustering(
        n_clusters=i, affinity="euclidean", linkage="average"
    )

    model.fit(hc_df)

    df["HC_segments"] = model.labels_

    print("~" * 40)

    sc = round(silhouette_score(hc_df, df["HC_segments"]), 3)

    sil_score.append(sc)

    print(
        "Silhouette Score for K=",
        i,
        ":",
        round(silhouette_score(hc_df, df["HC_segments"]), 3),
    )

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Silhouette Score for K= 2 : 0.57
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Silhouette Score for K= 3 : 0.516
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Silhouette Score for K= 4 : 0.475
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Silhouette Score for K= 5 : 0.44
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Silhouette Score for K= 6 : 0.415
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Silhouette Score for K= 7 : 0.418
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Silhouette Score for K= 8 : 0.343
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Silhouette Score for K= 9 : 0.342
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Silhouette Score for K= 10 : 0.346


plt.figure(figsize=(10, 5))
plt.plot(range(2, 11, 1), sil_score, marker="*")
plt.title("Sum of Intra Cluster Distance across different values of K")

plt.axvline(3, linestyle="--")

plt.xlabel("HC - clusters")
plt.ylabel("Silhouette Score")
plt.xticks(range(2, 11, 1))
plt.show()


#######
# We have identified the linkage and distance metric as euclidean and average
# Create dendrogram using these parameters at K = 3
#######

Z = linkage(hc_df, metric="euclidean", method="average")

plt.figure(figsize=(15, 10))
plt.title("Agglomerative Hierarchical Clustering Dendogram")
plt.xlabel("sample index")
plt.ylabel("Dendogram Distance")

dendrogram(
    Z,
    leaf_rotation=90.0,
    p=5,
    leaf_font_size=10,
    truncate_mode="level",
    color_threshold=3,
    link_color_func=lambda k: "purple",
)

# Threshold: we cut the tree at dendrogram distance of 3.2
plt.axhline(y=3.2, color="red")

plt.tight_layout()


%%time
HCmodel = AgglomerativeClustering(n_clusters=3, affinity="euclidean", linkage="average")
HCmodel.fit(hc_df)

CPU times: user 6.44 ms, sys: 860 µs, total: 7.3 ms
Wall time: 6.14 ms

AgglomerativeClustering(linkage='average', n_clusters=3)


# Add the cluster labels to the dataframes

df["HC_segments"] = HCmodel.labels_
df2["HC_segments"] = HCmodel.labels_
hc_df["HC_segments"] = HCmodel.labels_


display(hc_df.head())
display(df.head())


#####
# Code to visualise the Boxplots of numerical variables for each cluster obtained using Hierarchical Clustering
#####

fig, axes = plt.subplots(1, 5, figsize=(16, 6))
fig.suptitle(
    "Boxplot of numerical variables for each cluster obtained using Heirarchical Clustering",
    fontsize=20,
)

for ii in range(5):
    sns.boxplot(ax=axes[ii], y=df[df.columns[ii]], x=df["HC_segments"])


fig.tight_layout(pad=2.0)


# Map the HC number groups to match that of KM number groups
df["HC_segments"] = df["HC_segments"].map({0: 1, 1: 2, 2: 0})
hc_df["HC_segments"] = hc_df["HC_segments"].map({0: 1, 1: 2, 2: 0})


df["HC_Group"] = df["HC_segments"].apply(cluster_map)

pd.set_option("display.max_colwidth", None)

df.groupby("HC_Group").mean().round(2).reset_index().sort_values(
    by=["Total_Credit_Cards"], ascending=False
)


km_cluster_profile = df.groupby("K_means_segments").mean()


km_cluster_profile["Count_in_Each_Segment"] = (
    df.groupby("K_means_segments")["Avg_Credit_Limit"].count().values
)

# Exclude the HC_segments
column_to_exclude = "HC_segments"
km_cluster_profile = km_cluster_profile.loc[
    :, km_cluster_profile.columns != column_to_exclude
]


km_cluster_profile.style.highlight_max(color="lightgreen", axis=0).highlight_min(
    color="lightyellow"
)


hc_cluster_profile = df.groupby("HC_segments").mean()


hc_cluster_profile["Count_in_Each_Segment"] = (
    df.groupby("HC_segments")["Avg_Credit_Limit"].count().values
)

# Exclude the HC_segments
column_to_exclude = "K_means_segments"
hc_cluster_profile = hc_cluster_profile.loc[
    :, hc_cluster_profile.columns != column_to_exclude
]


hc_cluster_profile.style.highlight_max(color="lightgreen", axis=0).highlight_min(
    color="lightyellow"
)


# Assuming the two columns are named 'column1' and 'column2' in the DataFrame 'df'
unequal_rows = df[df["K_means_segments"] != df["HC_segments"]]
unequal_rows


record = data.iloc[313]
record

Sl_No                    314
Customer Key           72156
Avg_Credit_Limit        7000
Total_Credit_Cards         4
Total_visits_bank          2
Total_visits_online        2
Total_calls_made           4
Name: 313, dtype: int64


#####
# Boxplots to understand the customer distribution among the clusters for K_means Clustering.
#####
fig, axes = plt.subplots(1, 5, figsize=(16, 6))
fig.suptitle(
    "Boxplot of numerical variables for each cluster obtained using K-means Clustering",
    fontsize=20,
)

for ii in range(5):
    sns.boxplot(ax=axes[ii], y=df[df.columns[ii]], x=df["K_means_segments"])

fig.tight_layout(pad=2.0)


#####
# Boxplots to understand the customer distribution among the clusters for Hierarchical Clustering.
#####
fig, axes = plt.subplots(1, 5, figsize=(16, 6))
fig.suptitle(
    "Boxplot of numerical variables for each cluster obtained using Hierarchical Clustering",
    fontsize=20,
)

for ii in range(5):
    sns.boxplot(ax=axes[ii], y=df[df.columns[ii]], x=df["HC_segments"])

fig.tight_layout(pad=2.0)


# Bar plot for the mean values along the segments for K_means

km_df.groupby("K_means_segments").mean().plot.bar(figsize=(15, 6))

<AxesSubplot:xlabel='K_means_segments'>


# Bar plot for the mean values along the segments for HC

hc_df.groupby("HC_segments").mean().plot.bar(figsize=(15, 6))

<AxesSubplot:xlabel='HC_segments'>


#####
# Comparing output of both algorithms
#####

pd.crosstab(df["HC_Group"], df["K_means_Group"])


# getting the count for each unique value in Customer_Key
filtered = df2.groupby("Customer Key").count()

for i in filtered.loc[filtered.Sl_No >= 2].index:
    display(df2.loc[df2["Customer Key"] == i])

	Sl_No	Customer Key	Avg_Credit_Limit	Total_Credit_Cards	Total_visits_bank	Total_visits_online	Total_calls_made
0	1	87073	100000	2	1	1	0
1	2	38414	50000	3	0	10	9
2	3	17341	50000	7	1	3	4
3	4	40496	30000	5	1	1	4
4	5	47437	100000	6	0	12	3

	count	mean	std	min	25%	50%	75%	max
Sl_No	660.000	330.500	190.670	1.000	165.750	330.500	495.250	660.000
Customer Key	660.000	55141.444	25627.772	11265.000	33825.250	53874.500	77202.500	99843.000
Avg_Credit_Limit	660.000	34574.242	37625.488	3000.000	10000.000	18000.000	48000.000	200000.000
Total_Credit_Cards	660.000	4.706	2.168	1.000	3.000	5.000	6.000	10.000
Total_visits_bank	660.000	2.403	1.632	0.000	1.000	2.000	4.000	5.000
Total_visits_online	660.000	2.606	2.936	0.000	1.000	2.000	4.000	15.000
Total_calls_made	660.000	3.583	2.865	0.000	1.000	3.000	5.000	10.000

	Sl_No	Customer Key	Avg_Credit_Limit	Total_Credit_Cards	Total_visits_bank	Total_visits_online	Total_calls_made
411	412	50706	44000	4	5	0	2
541	542	50706	60000	7	5	2	2

	Avg_Credit_Limit	Total_Credit_Cards	Total_visits_bank	Total_visits_online	Total_calls_made
0	1.740	-1.249	-0.860	-0.547	-1.252
1	0.410	-0.788	-1.474	2.521	1.892
2	0.410	1.059	-0.860	0.134	0.146
3	-0.122	0.136	-0.860	-0.547	0.146
4	1.740	0.597	-1.474	3.202	-0.204

	pc1	pc2
0	0.566	0.520
1	-1.594	3.119
2	0.593	0.819
3	-0.247	0.134
4	1.661	3.660

	Sl_No	Customer Key	Avg_Credit_Limit	Total_Credit_Cards	Total_visits_bank	Total_visits_online	Total_calls_made
655	656	51108	99000	10	1	10	0
656	657	60732	84000	10	1	13	2
657	658	53834	145000	8	1	9	1
658	659	80655	172000	10	1	15	0
659	660	80150	167000	9	0	12	2

	Missing Count	% Missing
Sl_No	0	0.000000
Customer Key	0	0.000000
Avg_Credit_Limit	0	0.000000
Total_Credit_Cards	0	0.000000
Total_visits_bank	0	0.000000
Total_visits_online	0	0.000000
Total_calls_made	0	0.000000

	K_means_Group	Avg_Credit_Limit	Total_Credit_Cards	Total_visits_bank	Total_visits_online	Total_calls_made	K_means_segments
0	C1-OnlineBankers - Customers with: High Credit Limit & High Total Credit Cards	141040.000	8.740	0.600	10.900	1.080	2.000
1	C2-TelephoneBankers - Customers with: Moderate Credit Limit & Moderate Total Credit Cards	33782.380	5.520	3.490	0.980	2.000	1.000
2	C3-PhysicalBankers - Customers with: Low Credit Limit & Low Total Credit Cards	12174.110	2.410	0.930	3.550	6.870	0.000

	LinkageMethod	CopheneticCoefficient
0	single	0.739
1	complete	0.860
2	average	0.898
3	weighted	0.886
4	ward	0.742

	Avg_Credit_Limit	Total_Credit_Cards	Total_visits_bank	Total_visits_online	Total_calls_made	Count_in_Each_Segment
K_means_segments
0	12174.107143	2.410714	0.933036	3.553571	6.870536	224
1	33782.383420	5.515544	3.489637	0.981865	2.000000	386
2	141040.000000	8.740000	0.600000	10.900000	1.080000	50

	Avg_Credit_Limit	Total_Credit_Cards	Total_visits_bank	Total_visits_online	Total_calls_made	Count_in_Each_Segment
HC_segments
0	12197.309417	2.403587	0.928251	3.560538	6.883408	223
1	33713.178295	5.511628	3.485788	0.984496	2.005168	387
2	141040.000000	8.740000	0.600000	10.900000	1.080000	50

K_means_Group	C1-OnlineBankers - Customers with: High Credit Limit & High Total Credit Cards	C2-TelephoneBankers - Customers with: Moderate Credit Limit & Moderate Total Credit Cards	C3-PhysicalBankers - Customers with: Low Credit Limit & Low Total Credit Cards
HC_Group
C1-OnlineBankers - Customers with: High Credit Limit & High Total Credit Cards	50	0	0
C2-TelephoneBankers - Customers with: Moderate Credit Limit & Moderate Total Credit Cards	0	386	1
C3-PhysicalBankers - Customers with: Low Credit Limit & Low Total Credit Cards	0	0	223

	Sl_No	Customer Key	Avg_Credit_Limit	Total_Credit_Cards	Total_visits_bank	Total_visits_online	Total_calls_made
391	392	96929	13000	4	5	0	0
398	399	96929	67000	6	2	2	2

	Sl_No	Customer Key	Avg_Credit_Limit	Total_Credit_Cards	Total_visits_bank	Total_visits_online	Total_calls_made
104	105	97935	17000	2	1	2	10
632	633	97935	187000	7	1	7	0

AllLife Bank - Credit Card Customer Segmentation

General Overview¶

Background & Context¶

Objective¶

Key Questions:¶

Data Dictionary:¶

Sanity Checks¶

Importing Necessary Libraries¶

Loading the Dataset¶

Data Overview¶

Checking a few rows of the Dataset¶

Checking the shape of the dataset¶

Checking the Data Types & General Information of the Dataset¶

Getting the Statistical Summary for the Dataset¶

Checking Missing, Duplicate & Unique Values¶

Checking for Missing Values¶

Checking for Duplicate Records¶

Checking for Duplicate Value Entries¶

Exploratory Data Analysis¶

Univariate analysis¶

Bivariate analysis¶

Correlation Matrix¶

Insights Based on EDA¶

Data Preprocessing¶

Scaling¶

K-means Clustering¶

Checking Elbow Plot¶

Checking Silhouette Score¶

Silhouette Visualizer¶

We will create Silhouette Visualizer for K=3, 4 & 5¶

Intercluster Distance Visualizer¶

Checking the Intercluster Distance¶

We will proceed with K=3 for K-means Clustering¶

Creating Final Model¶

PCA - Principal Component Analysis¶

While we do not have a curse of dimentionality with this dataset, we will use PCA to reduce the features from 5 to 2 so that we can easily visualize it.¶

K-means Cluster Profiling¶

Hierarchical Clustering¶

We will now perform Hierarchical Clustering algorithm on the dataset and observe it's performance and clustering.¶

Cophenetic Coefficient¶

Checking Dendrograms¶

We Will Proceed with 3 Terminal Nodes for Hierarchical Clustering¶

Creating Final Model for Hierarchical Clustering¶

Hierarchical Cluster Profiling¶

Cluster Profiling and Comparison¶

Cluster Profiling: K-means Clustering¶

Cluster Profiling: Hierarchical Clustering¶

Finding the "misclassified" record¶

Comparison Continued...¶

Checking the Clusters of the Duplicated Values of Customer Key¶

Summary: K-means Clustering vs Hierarchical Clustering¶

Business Insights and Recommendations¶

Actionable Insights and Recommendations¶